package com.elasticsearch.jd_study.spider;


import com.elasticsearch.jd_study.pojo.Content;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;

//爬虫解析网页   这个 只能数据，电影和音乐 用tika
public class HtmlParseUtil {

    public static void main(String[] args) {
        try {
            ArrayList<Content> contents = new HtmlParseUtil().parseJD("ipad");
            contents.forEach(System.out::println);
        } catch (IOException e) {
            e.printStackTrace();
        }

    }


    public ArrayList<Content> parseJD(String keywords) throws IOException {

        ArrayList<Content> goodsList = new ArrayList<>();

        //获取请求
        String url = "https://list.tmall.com/search_product.htm?q="+keywords;

        //解析网页（Jsoup返回的就是网页  Document对象）
        Document document = Jsoup.parse(new URL(url), 30000);

        //所有在js中使用的方法 在这里都可以使用
        Element element = document.getElementById("J_ItemList");
        Elements product = element.getElementsByClass("product");

        //
        for (Element element1 : product) {

            Content content = new Content();

            Elements elementsByClass = element1.getElementsByClass("productImg-wrap");

            for (Element byClass : elementsByClass) {
                String img = byClass.getElementsByTag("img").attr("data-ks-lazyload");
                if (img.length()>0){
//                    img = img.replace("//","");
                    content.setImg(img);
                    System.out.println(img);
                }

            }

            String img = element1.getElementsByTag("img").attr("src");
            String productPrice = element1.getElementsByClass("productPrice").text();
            String productTitle = element1.getElementsByClass("productTitle").text();
            if (img.length()>0){
//                img = img.replace("//","");
                content.setImg(img);
                System.out.println(img);
            }
            content.setPrice(productPrice);
            content.setTitle(productTitle);
            System.out.println(productPrice);
            System.out.println(productTitle);
            System.out.println("---------good attr----------");

            goodsList.add(content);
        }
        return goodsList;

}
}
